  
package org.rescue.data.extraction.gate;
  
import java.util.*;
import java.io.*;
import java.net.*;

import org.rescue.search.Dataset;
import org.rescue.search.DatasetDB;
  
import gate.*;
import gate.creole.*;
import gate.util.*;
import gate.corpora.RepositioningInfo;
import gate.persist.SerialDataStore;

import gate.Document;

  
  /**
   * Extract place names from each document in a dataset.
   */
  public class ExtractGATEPlaces  {
  
    /** The corpus pipeline application to contain ANNIE */
    private SerialAnalyserController annieController;
  
    
    /**
     * Initialise the ANNIE system. This creates a corpus pipeline
     * application that can be used to run sets of documents through
     * the extraction system.
     */
    public void initAnnie() throws GateException {
      Out.prln("Initialising ANNIE...");
  
      // create a serial analyser controller to run ANNIE with
      annieController =
        (SerialAnalyserController) Factory.createResource(
          "gate.creole.SerialAnalyserController", Factory.newFeatureMap(),
          Factory.newFeatureMap(), "ANNIE_" + Gate.genSym()
        );
  
      // load each PR as defined in ANNIEConstants
      for(int i = 0; i < ANNIEConstants.PR_NAMES.length; i++) {
        FeatureMap params = Factory.newFeatureMap(); // use default parameters
        ProcessingResource pr = (ProcessingResource)Factory.createResource(ANNIEConstants.PR_NAMES[i], params);
  
        // add the PR to the pipeline controller
        annieController.add(pr);
      } // for each ANNIE PR
  
      Out.prln("...ANNIE loaded");
    } // initAnnie()
  
    
    
    /** Tell ANNIE's controller about the corpus you want to run on */
    public void setCorpus(Corpus corpus) {
      annieController.setCorpus(corpus);
    } 
  
    
    /** Run ANNIE */
    public void execute() throws GateException {
      Out.prln("Running ANNIE...");
      annieController.execute();
      Out.prln("...ANNIE complete");
    } 
  

    /** ensures temp directory for datasource exists and is empty */
    private String setupDataStoreDir() {
    	//TODO: update to handle multiple simultaneous extractions
    	File dir = new File("c:/temp/gate_ds_001");
    	if (dir.exists()) {
    		deleteDir(dir);
    	}
    	dir.mkdir();   	
    	return "file:///" + dir.getAbsolutePath();
    }
    
    /** deletes a directory and all its contents */
    private boolean deleteDir(File path) {    	
    	File[] files = path.listFiles();
    	for (int i=0; i<files.length; i++) {
    		if(files[i].isDirectory()) {
    			deleteDir(files[i]);
    		}
    		else {
    			files[i].delete();
    		}
    	}    	
    	return (path.delete());
    }
    
    
    /** extract geographical locations and insert into the document */
    public static void getPlaces(ArrayList<org.rescue.search.Document> documents)
    throws GateException, IOException {

    	// initialise the GATE library
    	try {
    		Gate.init();  
    	}
    	catch (GateException gex) {
    		Err.println("GATE Initialization Failed: " + gex.getMessage());
    		return;
    	}
   
    	try {
    		
	    	
	    	// Load ANNIE plugin
	    	File gateHome = Gate.getGateHome();
	    	File pluginsHome = new File(gateHome, "plugins");
	    	Gate.getCreoleRegister().registerDirectories(new File(pluginsHome, "ANNIE").toURL());
	    	Out.prln("...GATE initialised");
	 
	    	// initialise ANNIE 
	    	ExtractGATEPlaces annie = new ExtractGATEPlaces();
	    	annie.initAnnie();
	 
	  
	    	// create a datastore on disk
	    	String dsDir = annie.setupDataStoreDir();
	    	SerialDataStore sds = (SerialDataStore)Factory.createDataStore("gate.persist.SerialDataStore", dsDir);
	    	sds.open();
	    	
	 
	    	// create and save temp corpus into the datastore, then unload it
	    	Corpus tempCorpus = Factory.newCorpus("Temporary Corpus");
	    	Corpus sdsCorpus = (Corpus)sds.adopt(tempCorpus, null); 
	     	Factory.deleteResource(tempCorpus);
	    	tempCorpus = null;
	     	
	
	    	// iterate documents list and create GATE documents
	    	for(int i = 0; i < documents.size(); i++) {
	   		
	    		//create temp GATE document
	    		URL u = new URL(documents.get(i).getURL());       
	    		FeatureMap params = Factory.newFeatureMap();
	    		params.put("sourceUrl", u);
	    		params.put("preserveOriginalContent", new Boolean(true));
	    		params.put("collectRepositioningInfo", new Boolean(true));  
	    		Out.prln("Creating doc for " + u);
	    		Document doc = (Document)Factory.createResource("gate.corpora.DocumentImpl", params);
	    		doc.getFeatures().put("documentIndex", i);
	    		
	    		// save temp GATE document in the datastore, and then unload it 
	    		Document sdsDoc = (Document)sds.adopt(doc, null);
	    		Factory.deleteResource(doc);
	    		doc = null;
	    		
	    		// associate the GATE document with the corpus in the datastore, and then unload it
	    		sdsCorpus.add(sdsDoc);
	    		sdsCorpus.sync();
	    		Factory.deleteResource(sdsDoc);
	    		sdsDoc = null;
	    	} 
	
	 
	    	
	    	// tell the pipeline about the corpus and run it
	    	annie.setCorpus(sdsCorpus);
	    	annie.execute();
	    	
	    	
	 
	    	// for each document, get an XML document with the location names added
	    	Iterator iter = sdsCorpus.iterator();
	    	while(iter.hasNext()) {
	    		Document doc = (Document) iter.next();
	    		AnnotationSet defaultAnnotSet = doc.getAnnotations();
	    		Set annotTypesRequired = new HashSet();
	    		annotTypesRequired.add("Location");
	    		AnnotationSet peopleAndPlaces = defaultAnnotSet.get(annotTypesRequired);
	 
	    		FeatureMap features = doc.getFeatures();
	    		String originalContent = (String)features.get(GateConstants.ORIGINAL_DOCUMENT_CONTENT_FEATURE_NAME);
	    		RepositioningInfo info = (RepositioningInfo)features.get(GateConstants.DOCUMENT_REPOSITIONING_INFO_FEATURE_NAME);
	       
	       
	    		org.rescue.search.Document document = documents.get((Integer)features.get("documentIndex"));
	    		
	    		
	    		Iterator it = peopleAndPlaces.iterator();
	    		Annotation currAnnot;
	    		SortedAnnotationList sortedAnnotations = new SortedAnnotationList();
	 
	    		while(it.hasNext()) {
	    			currAnnot = (Annotation) it.next();
	    			sortedAnnotations.addSortedExclusive(currAnnot);
	    		}
	 
	    		StringBuffer editableContent = new StringBuffer(originalContent);
	    		long insertPositionEnd;
	    		long insertPositionStart;
	    		// insert anotation tags backward
	    		Out.prln("Unsorted annotations count: "+peopleAndPlaces.size());
	    		Out.prln("Sorted annotations count: "+sortedAnnotations.size());
	    		Out.prln("docId: " + document.getId());
	    		for(int i=sortedAnnotations.size()-1; i>=0; --i) {
	    			currAnnot = (Annotation) sortedAnnotations.get(i);
	    			insertPositionStart = currAnnot.getStartNode().getOffset().longValue();
	    			insertPositionStart = info.getOriginalPos(insertPositionStart);
	    			insertPositionEnd = currAnnot.getEndNode().getOffset().longValue();
	    			insertPositionEnd = info.getOriginalPos(insertPositionEnd, true);
	           
	    			String currAnnotText = originalContent.substring((int)insertPositionStart, (int)insertPositionEnd);
	    			String locType = (String)currAnnot.getFeatures().get("locType");
	    			Out.prln("->" + currAnnotText);
	    			
	    			Out.prln("locType: " + locType);
	    			
	    			document.addLocation(currAnnotText,locType);      
	    		}
	    	
	    		// clean up document
	    		doc.cleanup();
	    		sdsCorpus.unloadDocument(doc);
	    		Factory.deleteResource(doc);
	    		doc = null;
	    	} 
	    	
	    	// clean up datasource
	    	sds.close();
	    	sds.delete();
    	}
    	catch (GateException gex) {
    		Err.println("GATE Extraction Failed: " + gex.getMessage());
    		return;
    	}
    } 

    
    
    
    
    
    /**
     * test 
     */
    
    /*
    public static void main(String args[])
    throws GateException, IOException {

    		// add a java property for gate.home, maintaining original properties
    		Properties p = new Properties(System.getProperties());
    		p.setProperty("gate.home", "C:\\Program Files\\GATE-3.1");
    		System.setProperties(p);
    	
    		int datasetId = 22;//14; //22;
    		org.rescue.search.DatasetDB datasetDB = new DatasetDB();
    		org.rescue.search.Dataset d = datasetDB.readDataset(datasetId);
    	
    		getPlaces(d.getDocuments());
    		
    		Out.prln(d);
    		
   }
 */
    
   /**
    *
    */
   public static class SortedAnnotationList extends Vector {
     public SortedAnnotationList() {
       super();
     } // SortedAnnotationList
 
     public boolean addSortedExclusive(Annotation annot) {
       Annotation currAnot = null;
 
       // overlapping check
       for (int i=0; i<size(); ++i) {
         currAnot = (Annotation) get(i);
         if(annot.overlaps(currAnot)) {
           return false;
         } // if
       } // for
 
       long annotStart = annot.getStartNode().getOffset().longValue();
       long currStart;
       // insert
       for (int i=0; i < size(); ++i) {
         currAnot = (Annotation) get(i);
         currStart = currAnot.getStartNode().getOffset().longValue();
         if(annotStart < currStart) {
           insertElementAt(annot, i);
           /*
            Out.prln("Insert start: "+annotStart+" at position: "+i+" size="+size());
            Out.prln("Current start: "+currStart);
            */
           return true;
         } // if
       } // for
 
       int size = size();
       insertElementAt(annot, size);

       return true;
     } // addSorted
   } // SortedAnnotationList
 
}
 
